#!/usr/bin/env python
# File created on 04.07.08 by Tizian Schmidlin
# class created on 20.06.08
# class inspired on a programm by Peter SUNDSTROM
# FWLogParser class

import re, time, socket, array, string, os
from natsort import *
from resolving import ResolveIP
from parseutils import ParseUtils

class FWLogParser:
  def __init__(self, file = '', delimiter = ';'):
    # Initialize variables.
    
    self.Bytes = 0
    
    self.width = '36' # character width
    
    self.Header= []
    self.Lines = 0 # Processed lines
    self.Entries = 0 # Entries reported on
    self.Top = 20 # Size of the top ranking
    self.Domains = {} # Resolved domains hash
    self.UsrOrig = {}
    self.UsrSrc = {}
    self.UsrDst = {}
    
    self.p = ParseUtils()
    
    # Start and End date/time of the report
    self.StartDate = None
    self.EndDate = None
    self.StartTime = None
    self.EndTime = None
    
    # Start and End date/time of the matched entries
    self.StartMatchedReportDate = None
    self.EndMatchedReportDate = None
    self.StartMatchedReportTime = None
    self.EndMatchedReportTime = None
    
    self.Control = 0
    self.unknown = 0
    self.Ignored = 0
    
    self.Type = re.compile('reject|drop',re.IGNORECASE)
    self.re_type = '^(log|account|alert)$'
    self.re_ip = re.compile('\d+\.\d+\.\d+\.\d+')
    self.re_ignore = re.compile('^$')
    self.re_iip = re.compile('^(192\.168|10\.\d+|172\.\d+)\.\d+\.\d+') # Internal ips
    
    self.Ignore = True
    
    
    self.Attack = {}
    self.Attacks = 0
    self.AttackURL = {}
    self.AttackURLs = 0
    
    self.Alert = 0
    self.AlertType = {}
    
    self.OutTraffic = 0
    self.OutTraffic_Bytes = 0
    self.InTraffic = 0
    self.InTraffic_Bytes = 0
    
    self.Inbound = False
    self.Outbound = False    
    self.Svcname = True
    self.Svcport = True
    
    self.ExcludedServices = re.compile('^$')
    self.Exclude = {}
    
    self.XlateType = 'both'
    self.XlateAddr = 0
    self.XlatePort = 0
    
    self.ExcludeSrcServices = re.compile('^$')
    self.ExcludeSrc = {}
    
    self.ExcludeIF = re.compile('^$')
    self.ExcludeIFLines = 0
    
    self.SrcPortNum = False
    self.attackinfo = True
    
    self.Crypt = 0
    
    self.rtpattacks = False
    
    self.Matches = {} # Counter for entries based on keys
    
    self.t1 = time.time()
    self.t2 = self.t1
    
    self.file = file
    self.delimiter = delimiter
    
    self.Service = {}
    self.Rule = {}
    self.Source = {}
    self.Destination = {}
    self.Daily = {}
    self.FWhost = {}
    self.Network = {}
    
    self.Service_Bandwidth = {}
    self.Rule_Bandwidth = {}
    self.Source_Bandwidth = {}
    self.Destination_Bandwidth = {}
    self.Daily_Bandwidth = {}
    self.Network_Bandwidth = {}
    
    self.Clock = 24
    
    self.dnslist = {}
    
    self.Hourly_Hits = {}
    self.Hourly_Bandwidth = {}
    
    self.countries = {}
    self.CountryTranslation = {
      'AD':'Andorra',
      'AE':'United Arab Emirates',
      'AF':'Afghanistan',
      'AG':'Antigua and Barbuda',
      'AI':'Anguilla',
      'AL':'Albania',
      'AM':'Armenia',
      'AN':'Netherland Antilles',
      'AO':'Angola',
      'AQ':'Antarctica',
      'AR':'Argentina',
      'AS':'American Samoa',
      'AT':'Austria',
      'AU':'Australia',
      'AW':'Aruba',
      'AZ':'Azerbaidjan',
      'BA':'Bosnia-Herzegovina',
      'BB':'Barbados',
      'BD':'Banglades',
      'BE':'Belgium',
      'BF':'Burkina Faso',
      'BG':'Bulgaria',
      'BH':'Bahrain',
      'BI':'Burundi',
      'BJ':'Benin',
      'BM':'Bermuda',
      'BN':'Brunei Darussalam',
      'BO':'Bolivia',
      'BR':'Brazil',
      'BS':'Bahamas',
      'BT':'Buthan',
      'BV':'Bouvet Island',
      'BW':'Botswana',
      'BY':'Belarus',
      'BZ':'Belize',
      'CA':'Canada',
      'CC':'Cocos (Keeling) Isl.',
      'CF':'Central African Rep.',
      'CG':'Congo',
      'CH':'Switzerland',
      'CI':'Ivory Coast',
      'CK':'Cook Islands',
      'CL':'Chile',
      'CM':'Cameroon',
      'CN':'China',
      'CO':'Colombia',
      'CR':'Costa Rica',
      'CS':'Czechoslovakia',
      'CU':'Cuba',
      'CV':'Cape Verde',
      'CX':'Christmas Island',
      'CY':'Cyprus',
      'CZ':'Czech Republic',
      'DE':'Germany',
      'DJ':'Djibouti',
      'DK':'Denmark',
      'DM':'Dominica',
      'DO':'Dominican Republic',
      'DZ':'Algeria',
      'EC':'Ecuador',
      'EE':'Estonia',
      'EG':'Egypt',
      'EH':'Western Sahara',
      'ES':'Spain',
      'ET':'Ethiopia',
      'EU':'European Union',
      'FI':'Finland',
      'FJ':'Fiji',
      'FK':'Falkland Isl.(Malvinas)',
      'FM':'Micronesia',
      'FO':'Faroe Islands',
      'FR':'France',
      'FX':'France (European Ter.)',
      'GA':'Gabon',
      'GB':'Great Britain (UK)',
      'GD':'Grenada',
      'GE':'Georgia',
      'GH':'Ghana',
      'GI':'Gibraltar',
      'GL':'Greenland',
      'GP':'Guadeloupe (Fr.)',
      'GQ':'Equatorial Guinea',
      'GF':'Guyana (Fr.)',
      'GM':'Gambia',
      'GN':'Guinea',
      'GR':'Greece',
      'GT':'Guatemala',
      'GU':'Guam (US)',
      'GW':'Guinea Bissau',
      'GY':'Guyana',
      'HK':'Hong Kong',
      'HM':'Heard & McDonald Isl.',
      'HN':'Honduras',
      'HR':'Croatia',
      'HT':'Haiti',
      'HU':'Hungary',
      'ID':'Indonesia',
      'IE':'Ireland',
      'IL':'Israel',
      'IN':'India',
      'IO':'British Indian O. Terr.',
      'IQ':'Iraq',
      'IR':'Iran',
      'IS':'Iceland',
      'IT':'Italy',
      'JM':'Jamaica',
      'JO':'Jordan',
      'JP':'Japan',
      'KE':'Kenya',
      'KG':'Kirgistan',
      'KH':'Cambodia',
      'KI':'Kiribati',
      'KM':'Comoros',
      'KN':'St.Kitts Nevis Anguilla',
      'KP':'Korea (North)',
      'KR':'Korea (South)',
      'KW':'Kuwait',
      'KY':'Cayman Islands',
      'KZ':'Kazachstan',
      'LA':'Laos',
      'LB':'Lebanon',
      'LC':'Saint Lucia',
      'LI':'Liechtenstein',
      'LK':'Sri Lanka',
      'LR':'Liberia',
      'LS':'Lesotho',
      'LT':'Lithuania',
      'LU':'Luxembourg',
      'LV':'Latvia',
      'LY':'Libya',
      'MA':'Morocco',
      'MC':'Monaco',
      'MD':'Moldavia',
      'MG':'Madagascar',
      'MH':'Marshall Islands',
      'ML':'Mali',
      'MK':'Macedonia',
      'MM':'Myanmar',
      'MN':'Mongolia',
      'MO':'Macau',
      'MP':'Northern Mariana Isl.',
      'MQ':'Martinique (Fr.)',
      'MR':'Mauritania',
      'MS':'Montserrat',
      'MT':'Malta',
      'MU':'Mauritius',
      'MV':'Maldives',
      'MW':'Malawi',
      'MX':'Mexico',
      'MY':'Malaysia',
      'MZ':'Mozambique',
      'NA':'Namibia',
      'NC':'New Caledonia (Fr.)',
      'NE':'Niger',
      'NF':'Norfolk Island',
      'NG':'Nigeria',
      'NI':'Nicaragua',
      'NL':'Netherlands',
      'NO':'Norway',
      'NP':'Nepal',
      'NR':'Nauru',
      'NT':'Neutral Zone',
      'NU':'Niue',
      'NZ':'New Zealand',
      'OM':'Oman',
      'PA':'Panama',
      'PE':'Peru',
      'PF':'Polynesia (Fr.)',
      'PG':'Papua New',
      'PH':'Philippines',
      'PK':'Pakistan',
      'PL':'Poland',
      'PM':'St. Pierre & Miquelon',
      'PN':'Pitcairn',
      'PT':'Portugal',
      'PR':'Puerto Rico (US)',
      'PW':'Palau',
      'PY':'Paraguay',
      'QA':'Qatar',
      'RE':'Reunion (Fr.)',
      'RO':'Romania',
      'RS':'Serbia',
      'RU':'Russian Federation',
      'RW':'Rwanda',
      'SA':'Saudi Arabia',
      'SB':'Solomon Islands',
      'SC':'Seychelles',
      'SD':'Sudan',
      'SE':'Sweden',
      'SG':'Singapore',
      'SH':'St. Helena',
      'SI':'Slovenia',
      'SJ':'Svalbard & Jan Mayen Is',
      'SK':'Slovak Republic',
      'SL':'Sierra Leone',
      'SM':'San Marino',
      'SN':'Senegal',
      'SO':'Somalia',
      'SR':'Suriname',
      'ST':'St. Tome and Principe',
      'SU':'Soviet Union',
      'SV':'El Salvador',
      'SY':'Syria',
      'SZ':'Swaziland',
      'TC':'Turks & Caicos Islands',
      'TD':'Chad',
      'TF':'French Southern Terr.',
      'TG':'Togo',
      'TH':'Thailand',
      'TJ':'Tadjikistan',
      'TK':'Tokelau',
      'TM':'Turkmenistan',
      'TN':'Tunisia',
      'TO':'Tonga',
      'TP':'East Timor',
      'TR':'Turkey',
      'TT':'Trinidad & Tobago',
      'TV':'Tuvalu',
      'TW':'Taiwan',
      'TZ':'Tanzania',
      'UA':'Ukraine',
      'UG':'Uganda',
      'UK':'United Kingdom',
      'UM':'US Minor outlying Isl.',
      'US':'United States',
      'UY':'Uruguay',
      'UZ':'Uzbekistan',
      'VA':'Vatican City State',
      'VC':'St.Vincent & Grenadines',
      'VE':'Venezuela',
      'VG':'Virgin Islands (British)',
      'VI':'Virgin Islands (US)',
      'VN':'Vietnam',
      'VU':'Vanuatu',
      'WF':'Wallis & Futuna Islands',
      'WS':'Samoa',
      'YE':'Yemen',
      'YU':'Yugoslavia',
      'ZA':'South Africa',
      'ZM':'Zambia',
      'ZR':'Zaire',
      'ZW':'Zimbabwe',
      'ARPA':'Old style Arpanet',
      'COM':'US Commercial',
      'EDU':'US Educational',
      'GOV':'US Government',
      'INT':'International',
      'MIL':'US Military',
      'NATO':'Nato field',
      'NET':'Network',
      'ORG':'Non-Profit',
      'INFO':'Info',
      'BIZ':'Business',
      'GBL':'MSNBot',
      'NAME':'Individual',
      'MUSEUM':'Museum',
      'PRO':'Professionals',
      'AERO':'Air-transport',
      'COOP':'Cooperatives',
      'CSIRO.AU':'Australian Research',
      'COM.AU':'Australian Commercial',
      'NET.AU':'Australian Network',
      'GOV.AU':'Australian Government',
      'EDU.AU':'Australian Educational',
      'ORG.AU':'Australian Non-Profit',
      'CO.NZ':'New Zealand Commercial',
      'NET.NZ':'New Zealand Network',
      'MOBI':'dotMobi',
      'UNKNOWN': 'Unknown'
    }
    r = ''
    for ext in self.CountryTranslation:
      r += "%s|" % ext
    r.rstrip("|")
    self.re_domains = re.compile('(%s)$'%r)
    
  def run(self):
    self.analyse()
  def __str__(self):
    r_string = """
    Report generated on: %s
    Period for report data:%s at %s to %s at %s
    Total entries processed : %d
    Inbound traffic :         %d
    Outbound traffic :        %d
    Control messaged :        %d
    Alert entries :           %d
    Encrypt/Decrypt entries : %d
    Unknown entries :         %d
    Entries Ignored :         %d
    Attack types :            %d
    Unique Attack URLs :      %d
    
    SUMMARY INFORMATION
    %s
    
    TOP 20 DNS:
    %s
    
    Top access stats :
    ==================
    
    Top services :
    %s
    
    Top sources :
    %s
    
    Top destinations :
    %s
    
    Top rules :
    %s
    
    Daily accesses :
    %s
    
    Top FWhost :
    %s
    
    Top Network Connections :
    %s """ % (str(time.ctime()),self.StartDate, self.StartTime, self.EndDate, self.EndTime,
                                       self.Lines, self.Inbound, self.Outbound, self.Control, self.Alert,
                                       self.Crypt, self.unknown, self.Ignored, self.Attacks, self.AttackURLs,
                                       self.Summary('Attack'), self.topdomains(),self.topvalues('Service'),
                                       self.topvalues('Source'), self.topvalues('Destination'), self.topvalues('Rule'), 
                                       self.topvalues('Daily'), self.topvalues('FWhost'), self.topvalues('Network'))
    
    
    if self.topbandwidth('Service'):
      r_string += """
    Top bandwidth stats :
    =====================
    
    Top service bandwidth:
    %s
    
    Top source bandwidth:
    %s
    
    Top destination bandwidth:
    %s
    
    Top rule bandwidth:
    %s
    
    Top daily bandwidth:
    %s
    
    Top network bandwidth:
    %s
    """ % (self.topbandwidth('Service'), self.topbandwidth('Source'),self.topbandwidth('Destination'),
             self.topbandwidth('Rule'),self.topbandwidth('Daily'), self.topbandwidth('Network'))
    
    r_string += """
    Traffic :
    =========
    %17s : %d
    %17s : %d
    
    %17s : %d
    %17s : %d""" % ('Inbound Traffic', self.InTraffic, 'Bandwidth', self.InTraffic_Bytes,
                    'Oubount Traffic', self.OutTraffic, 'Bandwidth', self.OutTraffic_Bytes)
    
    r_string += """
    Hourly stats:
    =============
    Hits in time span :
    %s
    
    Bytes in time span :
    %s
    """ % (self.hitsintimespan(), self.bytesintimespan())
    
    r_string += """
    Processing time : %f seconds""" % (time.time()-self.t1)
    return r_string
  
  def __iadd__(self, other):
    print other.__class__, self.__class__
    if other.__class__ is self.__class__:
      self.Lines += other.Lines # Processed lines
      self.Entries = other.Entries # Entries reported on
      
      u = ParseUtils()
      
      u.mergedict(self.Domains, other.Domains)
      u.mergedict(self.UsrOrig, other.UsrOrig)
      u.mergedict(self.UsrSrc, other.UsrSrc)
      u.mergedict(self.UsrDst, other.UsrDst)
      
      # Initialize the values so they can be compared
      #ssdate = u.splitdate(self.StartDate)
      #sedate = u.splitdate(self.EndDate)
      #sstime = u.splittime(self.StartTime)
      #setime = u.splittime(self.EndTime)
      
      osdate = u.splitdate(other.StartDate)
      oedate = u.splitdate(other.EndDate)
      ostime = u.splittime(other.StartTime)
      oetime = u.splittime(other.EndTime)
      
      # Keep only the extrema infos
      if self.StartDate != None and u.splitdate(self.StartDate) > osdate:
        self.StartDate = other.StarDate
        self.StartTime = other.StartTime
      
      if self.EndDate != None and u.splitdate(self.EndDate) < osdate:
        self.EndDate = other.EndDate
        self.EndTime = other.EndTime
      
      # Do the same as before with the matched data entries
      
      #ssdate = u.splitdate(self.StartMatchedReportDate)
      #sedate = u.splitdate(self.EndMatchedReportDate)
      #sstime = u.splittime(self.StartMatchedReportTime)
      #setime = u.splittime(self.EndMatchedReportTime)
      
      osdate = u.splitdate(other.StartMatchedReportDate)
      oedate = u.splitdate(other.EndMatchedReportDate)
      ostime = u.splittime(other.StartMatchedReportTime)
      oetime = u.splittime(other.EndMatchedReportTime)
      
      # Keep only the extrema infos
      if self.StartMatchedReportDate != None and u.splitdate(self.StartMatchedReportDate) > osdate:
        self.StartMatchedReportDate = other.StarMatchedReportDate
        self.StartMatchedReportTime = other.StarMatchedReportTime
      
      if self.EndMatchedReportDate != None and u.splitdate(self.EndMatchedReportDate) < osdate:
        self.EndMatchedReportDate = other.EndMatchedReportDate
        self.EndMatchedReportTime = other.EndMatchedReportTime
      
      self.Control += other.Control
      self.unknown += other.unknown
      self.Ignored += other.Ignored
      
      u.mergedict(self.Attack, other.Attack)
      self.Attacks += other.Attacks
      
      u.mergedict(self.AttackURL, other.AttackURL)
      self.AttackURLs += other.AttackURLs
      
      u.mergedict(self.AlertType, other.AlertType)
      self.Alert += other.Alert
      
      self.OutTraffic = other.OutTraffic
      self.OutTraffic_Bytes = other.OutTraffic_Bytes
      self.InTraffic = other.InTraffic
      self.InTraffic_Bytes = other.InTraffic_Bytes
      
      u.mergedict(self.Exclude, other.Exclude)
      
      self.XlateAddr += other.XlateAddr
      self.XlatePort += other.XlatePort
      
      u.mergedict(self.ExcludeSrc, other.ExcludeSrc)
      
      self.ExcludeIFLines += other.ExcludeIFLines
      
      self.Crypt += other.Crypt
      
      u.mergedict(self.Matches, other.Matches)
        
      if self.t1 > other.t1:
        self.t1 = other.t1
      
      if self.t2 < other.t2:
        self.t2 = other.t2
      
      u.mergedict(self.Service, other.Service)
      u.mergedict(self.Rule, other.Rule)
      u.mergedict(self.Source, other.Source)
      u.mergedict(self.Destination, other.Destination)
      u.mergedict(self.Daily, other.Daily)
      u.mergedict(self.FWhost, other.FWhost)
      u.mergedict(self.Network, other.Network)
      
      u.mergedict(self.Service_Bandwidth, other.Service_Bandwidth)
      u.mergedict(self.Rule_Bandwidth, other.Rule_Bandwidth)
      u.mergedict(self.Source_Bandwidth, other.Source_Bandwidth)
      u.mergedict(self.Destination_Bandwidth, other.Destination_Bandwidth)
      u.mergedict(self.Daily_Bandwidth, other.Daily_Bandwidth)
      #u.mergedict(self.FWhost_Bandwidth, other.FWhost_Bandwidth)
      u.mergedict(self.Network_Bandwidth, other.Network_Bandwidth)
    else:
      raise 'Error! You cannot add this element with a non %s element' % self.__class__
    
  def Summary(self, column, title = ''):
    # copie-suppression
    columntitle = column.replace('_Bandwith', '')
    
    if title == '' : title = column+' entries: Top '+str(self.Top)
    
    data = []
    datakeys = []
    a = {}
    #a.iteritems()
    a = getattr(self,column)
    for key, value in a.iteritems():
      line = str(key) + " : " + str(value)
      
    return line
  
  def analyse(self):
    #file = self.file
    if os.path.isfile(self.file):
      input = file(self.file, 'r')
      print "proceeding file %s " % str(self.file)
      x = ''
      head = "num%sdate%stime" % (self.delimiter, self.delimiter)
      if self.t1 == None:
        self.t1 = time.time()
      for x in input:
        # delete the last character
        x = x[:-1]
        if re.compile('^$').match(x,1) == None:
          if x.find(head) >= 0:
            self.Header = x.split(self.delimiter)
            self.Lines += 1
            continue
          else:
            # parsing
            self.parse(x)
            #job_server.submit(self.parse, (x,))
            self.Lines += 1
        if self.Lines % 10000 == 0:
          print "Processed 10000 lines in %f seconds, matched %d entries in file %s" % (time.time()-self.t2, self.Entries, self.file)
          self.t2 = time.time()
      input.close()
    else:
      print "%s is not a valid file path!" % self.file
  
  def __len__(self):
    return self.Lines
  
  def parse(self, x):
    LogLine = []
    LogLine = x.split(self.delimiter)
    Field = 0
    
    logentry = {}
    
    # Create a hash based on the header fieldname
    for Log in LogLine:
      logentry[self.Header[Field]] = Log
      Field += 1
    
    # Save start and end dates for the report
    
    if str(self.StartDate) == 'None':
      self.StartDate = "%09s" % logentry['date']
    else:
      self.EndDate = "%09s" % logentry['date']
    
    if self.StartTime == None:
      self.StartTime = logentry['time']
    else:
      self.EndTime = logentry['time']
    
    # Ignore "control" messages
    if logentry['type'] == 'control':
      self.Control += 1
      return
    
    # Ignore any non standard log entries
    if re.search(self.re_type,logentry['type'], 1) == None:
      self.unknown += 1
      return
    
    # Skip any non alert entries if the report type is only alerts
    if self.rtpattacks and logentry['product'] != 'SmartDefense':
      self.Ignored += 1
      return
    
    # Short logging does not include the rule number.
    # Only relevant for FW1 versions 1.x-4.x
    
    if logentry['rule'] == '' and self.Type.search(logentry['rule'], 1):
      logentry['rule'] = 'n/a'
      logentry['s_port'] = 'n/a'
    
    if logentry.has_key('bytes'):
      self.Bytes += int(logentry['bytes'])
    
    #
    # Process attack info from SmartDefense
    #
    
    if logentry['attack'] is None or logentry['attack'] == '':
      logentry['attack'] = 'n/a'
    else:
      self.Attack.setdefault(logentry['attack'], 0)
      self.Attack[logentry['attack']] += 1
      self.Attacks += 1
      if logentry.has_key('URL filter pattern detected'):
        self.AttackURL.setdefault(logentry['attack'], 0)
        self.AttackURL[logentry['URL filter pattern detected']] += 1
        self.AttackURLs += 1
    
    # Process any alert entries. 
    #
    # If the alert has miminal detail (eg: spoofalert), just count it
    # and go to the next entry.
    
    if logentry['type'].find('alert') >= 0:
      self.Alert += 1
      self.AlertType.setdefault(logentry['alert'], 0)
      self.AlertType[logentry['alert']] += 1
      if logentry['src'] is None:
        return
    
    # Count encrypt/decrypt entries
    
    if logentry['action'] == 'crypt':
      self.Crypt += 1
    
    # Breakup inbound and outbound traffic
    
    if logentry['i/f_dir'] == 'outbound':
      self.OutTraffic += 1
      if logentry['type'] == 'account':
        self.OutTraffic_Bytes += logentry['bytes']
      if self.Inbound:
        self.Ignored += 1
        return
    if logentry['i/f_dir'] == 'inboud':
      self.InTraffic += 1
      if logentry['type'] == 'account':
        self.InTraffic_Bytes += logentry['bytes']
      if self.Outbound :
        self.Ignored += 1
        return
    
    
    # Convert service port to name if specified
    if self.Svcname:
      if logentry['proto'] != 'icmp':
        if re.compile('\d').search(logentry['service']) != None:
          logentry['service'] = self.SvcName(logentry['service'], logentry['proto'])
        if re.compile('\d').search(logentry['s_port']) != None:
          logentry['s_port'] = self.SvcName(logentry['s_port'], logentry['proto'])
    
    
    # Convert service name to port if specified
    if self.Svcport:
      if logentry['service'] != 'icmp':
        if re.compile('[A-Za-z]').search(logentry['service']) != None:
          logentry['service'] = self.SvcPort(logentry['service'], logentry['proto'])
        if re.compile('[A-Za-z]').search(logentry['s_port']) != None:
          logentry['s_port'] = self.SvcPort(logentry['s_port'], logentry['proto'])
          
    # Exclude specified services, if specified
    if self.ExcludedServices:
      if self.ExcludedServices.search(logentry['service']):
        self.Ignored += 1
        self.Exclude.setdefault(logentry['service'], 0)
        self.Exclude[logentry['service']] += 1
        return
    
    # Convert src/dst address/port to use translated address/port
    if self.XlateType and logentry['xlatesrc']:
      if self.XlateType == 'both':
        if logentry['src'] != logentry['xlatesrc'] : logentry['src'] += "/"+logentry['xlatesrc']
        if logentry['dst'] != logentry['xlatedst'] : logentry['dst'] += "/"+logentry['xlatedst']
        if logentry['service'] != logentry['xlatedport'] : logentry['service'] += "/"+logentry['xlatedport']
        if logentry['s_port'] != logentry['xlatesport'] : logentry['s_port'] += "/"+logentry['xlatesport']
      elif self.XlateType == 'translate':
        logentry['src'] = logentry['xlatesrc']
        logentry['dst'] = logentry['xlatedst']
        logentry['service'] = logentry['xlatedport']
        logentry['s_port'] = logentry['xlatesport']
    
    if logentry['action'] == 'drop' and self.Type.match(logentry['action']) or \
    logentry['action'] == 'reject' and self.Type.match(logentry['action']) or \
    re.compile('accept|crypt').search(logentry['action'],1):
      return
    
    
    # Exclude specified source services, if specified
    if self.ExcludeSrcServices:
      if self.ExcludeSrcServices.search(logentry['s_port']):
        self.Ignored += 1
        self.ExcludeSrc.setdefault(logentry['s_port'], 0)
        self.ExcludeSrc[logentry['s_port']] += 1
        return
    # Keep all 
    
    # Create the key hash with the matched data
    #print logentry
    key = self.CreateKey(logentry)
    
    #######
    #print key
    # INCOMPLETE!!! MAY BE MOVED
    # Ignore any specified entries
    if self.Ignore:
      if self.re_ignore.search(key) == None:
        pass
      else:
        return
    
    # Ignore any specified interfaces
    if self.ExcludeIF:
      if self.ExcludeIF.search(logentry['i/f_name']):
        self.Ignored += 1
        self.ExcludeIFLines += 1
        return
    
    # Save Start and End date for matched Data
    if self.StartMatchedReportDate == None:
      self.StartMatchedReportDate = "%09s" % logentry['date']
    else:
      self.EndMatchedReportDate = "%09s" % logentry['date']
    
    # Save Start and End time for matched Data
    if self.StartMatchedReportTime == None:
      self.StartMatchedReportTime = logentry['time']
    else:
      self.EndMatchedReportTime = logentry['time']
    
    self.Entries += 1
    
    self.Matches.setdefault(key, 0)
    self.Matches[key] += 1 # Increment counter for entries based on keys
    
    # Increment counter for Translated addresses/ports
    if logentry['xlatesrc']:
      self.XlateAddr += 1
      if logentry['service'] != logentry['xlatedport'] or logentry['s_port'] != logentry['xlatesport']:
        self.XlatePort += 1
    
    RuleNo = logentry['rule']
    SourceName = logentry['src']
    DestinationName = logentry['dst']
    DateName = logentry['date']
    
    if logentry['proto'] == 'icmp':
      ServiceType = "%s/%s" % (logentry['icmp-type'], logentry['icmp-code'])
    else:
      ServiceType = logentry['service']
    
    # Service entry for Service count
    s = "%s(%s)" % (logentry['proto'], ServiceType)
    self.Service.setdefault(s, 0)#.increment(1)#  += 1
    self.Service[s] += 1
    
    self.Rule.setdefault(RuleNo, 0)#.increment(1)# += 1
    self.Rule[RuleNo] += 1
    
    self.Source.setdefault(SourceName, 0)#.increment(1)# += 1
    self.Source[SourceName] += 1
    
    self.Destination.setdefault(DestinationName, 0)#.increment(1)# += 1
    self.Destination[DestinationName] += 1
    
    self.Daily.setdefault(DateName, 0)#.increment(1)# += 1
    self.Daily[DateName] += 1
    self.FWhost.setdefault(logentry['orig'], 0)#.increment(1)# += 1
    self.FWhost[logentry['orig']] += 1
    
    Interface = "%s %s" % (logentry['orig'], logentry['i/f_name'])
    n = "%s (%s)" % (Interface, logentry['i/f_dir'])
    self.Network.setdefault(n, 0)
    self.Network[n] += 1
    
    # Only applicable to account logs
    if logentry['type'] == 'account':
      bytes = logentry['bytes']
      self.Bytes += bytes
      self.FWhost_Bandwidth.setdefault(logentry['orig'], 0)
      self.FWhost_Bandwidth[logentry['orig']] += bytes
      self.Rule_Bandwidth.setdefault(RuleNo, 0)
      self.Rule_Bandwidth[RuleNo] += bytes
      self.Source_Bandwidth.setdefault(SourceName, 0)
      self.Source_Bandwidth[SourceName] += bytes
      self.Destination_Bandwidth.setdefault(DestinationName, 0)
      self.Destination_Bandwidth[DestinationName] += bytes
      self.Service_Bandwidth.setdefault(ServiceName, 0)
      self.Service_Bandwidth[ServiceName] += bytes
      self.Daily_Bandwidth.setdefault(DateName, 0)
      self.Daily_Bandwidth[DateName] += bytes
      self.Network_Bandwidth.setdefault(n, 0)
      self.Network_Bandwidth[n] += bytes
    
    Hour = logentry['time'].split(':')
    Duration =  str(Hour[0])
    
    self.Hourly_Hits.setdefault(Duration, 0)
    self.Hourly_Hits[Duration] += 1
    
    self.Hourly_Bandwidth.setdefault(Duration, 0)
    if logentry.has_key('bytes'):
      self.Hourly_Bandwidth[Duration] += logentry['bytes']
    
  def resolve(self):
    pinglist = []
    #print self.UsrOrig
    t1 = time.time()
    print "Waiting for all resolutions to finish..."
    
    i = 0
    for ip in self.UsrOrig:
      current = ResolveIP(ip)
      pinglist.append(current)
      current.start()
      
    for ip in self.UsrSrc:
      current = ResolveIP(ip)
      pinglist.append(current)
      current.start()
      
    for ip in self.UsrDst:
      current = ResolveIP(ip)
      pinglist.append(current)
      current.start()
    
    resolved = 0
    
    # Wait until every Thread is terminated and then analyze the result
    processed = 0
    total_addresse = len(pinglist)
    unresolved_types = {'Unknown Host': 0, 'Host name lookup failure' :0}
    for ip in pinglist:
      #print ip.ip
      ip.join()
      
      if ip.unresolved.has_key(1) : unresolved_types['Unknown Host'] += 1
      if ip.unresolved.has_key(2) : unresolved_types['Host name lookup failure'] += 1
        
      #unresolved += ip.unresolved
      resolved += ip.resolved
      self.dnslist.setdefault(ip.host, 0)
      self.dnslist[ip.host] += 1
      country = ip.host.split('.')
      try:
        self.countries.setdefault(self.CountryTranslation[country[-1].upper()], 0)
        self.countries[self.CountryTranslation[country[-1].upper()]] += 1
      except:
        print country, ip.completehost, ip.ip
      processed += 1
      if processed % 1000 == 0:
        print "Resolved 1000 more addresses, %d in total, %d left" % (processed, total_addresse-processed)
    unresolved = unresolved_types['Unknown Host'] + unresolved_types['Host name lookup failure']
    print """
    ==============================================
    Analysed %d addresses
    %d resolved addresses
    %d unresolved addresses
      Unresolved address errors :
        Unknown host:               %d
        Host name lookup failures:  %d
    ==============================================""" % (int(resolved+unresolved), int(resolved), unresolved, unresolved_types['Unknown Host'], unresolved_types['Host name lookup failure'])
    #for ping in pinglist:
      #print ping.host
      #print ping.aliaslist
    
  def SvcName(self, port, proto):
    #name = os.popen('getsrvbyport '+str(port)+' '+str(proto))
    #for n in name:
    #  print n
    #if not name :
    name = proto
    return name
  
  def SvcPort(self, host, proto):
    #name = os.popen('getsrvbyname '+str(host)+' '+str(proto))
    #for n in name:
    #  print n
    #if not name :
    name = host
    return name
  
  # INCOMPLETE!!!!
  def CreateKey(self, logentry):
    
    self.checkAndInitIP('UsrOrig', logentry['orig'])
    self.checkAndInitIP('UsrSrc', logentry['src'])
    self.checkAndInitIP('UsrDst', logentry['dst'])
    
    # Convert the FW host origin to shortname in uppercase
    
    logentry.setdefault('dst', 'No-address')
    logentry.setdefault('src', 'No-address')
    logentry.setdefault('service', 'No-service')
    logentry.setdefault('s_port', 'No-service')
    
    if self.SrcPortNum:
      if logentry['proto'] == 'icmp':
        key = logentry['orig']+"\t"+logentry['src']+'('+logentry['s_port']+')'+"\t"+logentry['dst']+"\t"+logentry['proto']+'('+logentry['icmp-type']+'/'+logentry['icmp-code']+")\t"+logentry['rule']
      else:
        key = logentry['orig']+"\t"+logentry['src']+'('+logentry['s_port']+')'+"\t"+logentry['dst']+"\t"+logentry['proto']+'('+logentry['service']+")\t"+logentry['rule']
    else:
      if logentry['proto'] == 'icmp':
        key = logentry['orig']+"\t"+logentry['src']+"\t"+logentry['dst']+"\t"+logentry['proto']+'('+logentry['icmp-type']+'/'+logentry['icmp-code']+")\t"+logentry['rule']
      else:
        key = logentry['orig']+"\t"+logentry['src']+"\t"+logentry['dst']+"\t"+logentry['proto']+'('+logentry['service']+")\t"+logentry['rule']
    logentry.setdefault('alert', 'noalert')
    key += "\t"+logentry['alert'] + "\t" + logentry['action']
    
    if self.attackinfo:
      if logentry['attack']:
        key += "\t"+logentry['attack']
      else:
        key += "\tn/a"
    return key
  
  def checkAndInitIP(self, attr, ip):
    if self.re_ip.search(ip):
      getattr(self, attr).setdefault(ip, 0)
      getattr(self, attr)[ip] += 1
    else:
      print ip
  def topdomains(self):
    
    i = 0
    j = 0    
    a = self.countries.items()
    a.sort(self.p.sortfunc)
    #print a
    
    r_string = '''
    +----+-------------------------------------+------------+
    | No |                        Domaine name | Occurences |
    +----+-------------------------------------+------------+
    '''
    c = []
    while(1):
      #if a[i][0] != '':
      if a[i][0] != 'Unknown':
        r_string += ("""| %2d | %"""+self.width+"""s | %10s |
    """) % (j, str(a[i][0]),str(a[i][1]))
        j += 1
        c.append((j, str(a[i][0]), a[i][1]))
      i += 1
      if i > self.Top : break
      if j > len(a) : break
    r_string += '+----+--------------------------------------+------------+'
    r_string += self.colstats(c)
    
    return r_string
  
  def top(self, element):
    a = getattr(self, element).items()
    a.sort(self.p.sortfunc)
    b = []
    i = 0
    #print a
    while(1):
      if i >= len(a) : break
      if a[i][0] != 'Unknown':
        b.append(a[i])
      i += 1
      if i > self.Top: break
    a = []
    return b
  
  def topips(self, element):
    a = getattr(self, element).items()
    a.sort(self.p.sortfunc)
    b = []
    i,j = 0,0
    while(1):
      if self.re_iip.search(a[i][0]) == None:
        b.append(a[i])
        j += 1
      i += 1
      if i >= len(a) : break
      if j > self.Top : break
    a = []
    return b
  
  def topvalues(self, element):
    r_string = ''
    topdest = self.topips(element)
    
    i = 0
    r_string = ("""
    +----+-------------------------------------+------------+
    | No | %"""+self.width+"""s | %10s |
    +----+-------------------------------------+------------+
    """) % (element, 'Occurences')
    c = []
    i = 1 
    for dst in topdest:
      # Filter internal ip addresses
      r_string += ("""| %2d | %"""+self.width+"""s | %10s |
    """) % (i, str(dst[0]), str(dst[1]))
      c.append((i, str(dst[0]), dst[1]))
      i += 1
    r_string += '+----+-------------------------------------+------------+'
    r_string += self.colstats(c)
    topdest = []
    return r_string
  
  def topbandwidth(self, element):
    a = self.top(str(element)+"_Bandwidth")
    r_string = ("""
    +----+--------------------------------------+------------+
    | No | %"""+self.width+"""s | %10s |
    +----+--------------------------------------+------------+
    """) % ('Name', 'Bandwidth')
    c = []
    i = 1
    for b in a:
      r_strint += ("""| %2d | %"""+self.width+"""s | %10s |
    """) % ( i, b[0], b[1])
      c.append((i, b[0], b[1]))
      i += 1
    r_string += '+----+--------------------------------------+------------+'
    r_string += self.colstats(c)
    return r_string
  
  def hitsintimespan(self):
    r_string = ("""
    +----+--------------------------------------+------------+
    | No | %"""+self.width+"""s | %10s |
    +----+--------------------------------------+------------+
    """) % ('Time Span', 'Hits')
    a = natsorted(self.Hourly_Hits.keys())
    c = []
    i = 1
    for b in a:
      h = "%02d:00-%02d:00" % (int(b), int(b)+1)
      r_string += ("""| %2d | %"""+self.width+"""s | %10d |
    """) % (i, h, self.Hourly_Hits[b])
      c.append((i, h, self.Hourly_Hits[b]))
      i += 1
    r_string += "+----+--------------------------------------+------------+"
    r_string += self.colstats(c)
    return r_string
  
  def bytesintimespan(self):
    r_string = ("""
    +----+--------------------------------------+------------+
    | No | %"""+self.width+"""s | %10s |
    +----+--------------------------------------+------------+
    """) % ('Time Span', 'Bandwidth')
    a = natsorted(self.Hourly_Bandwidth.keys())
    c = []
    i = 1
    for b in a:
      h = "%02d:00-%02d:00" % (int(b), int(b)+1)
      r_string += ("""| %2d | %"""+self.width+"""s | %10d |
    """) % (i, h, self.Hourly_Bandwidth[b])
      c.append((i, h, self.Hourly_Bandwidth[b]))
      i += 1
    r_string += '+----+--------------------------------------+------------+'
    r_string += self.colstats(c)
    return r_string
  
  def colstats(self, a):
    r_string = "\n    +----+-------------------------------------------------------------+\n"
    total = 0
    if str(type(a))== "<type 'list'>":
      if len(a) > 0 and str(type(a[0])) == "<type 'tuple'>":
        for b, c, d in a:
          total += d
      elif len(a) > 0:
        for b in a:
          total += int(b)
      else:
        r_string += "    | No | values                                                      |\n"
    elif str(type(a)) == "<type 'dict'>":
      for b in a:
        total += a[b]
    step = total / 60
    
    if str(type(a)) == "<type 'list'>" and step > 0:
      for b, c, d in a:
        r_string += "    | " + ("%2d" % b) +" | " + "="*int(int(d) / step) + '-'*int(60 - int(d) / step) + "|\n"
    elif str(type(a)) == "<type 'dict'>" and step > 0:
      for b in a:
        r_string += "    |" + "="*int(int(c) / step) + '-'*int(60 - int(c) / step) + "|\n"
    return r_string + "    +----+-------------------------------------------------------------+\n"